------------------------------------------------------------------------------
-- FILE: ~reissing/prokurs/Blatt6/Ada_Tokenizer.adb
--
--   Copyright (c) 2000 Uni Stgt.
--   All rights reserved..
--
-- PROJECT:  Programmierkurs, Blatt 6
-- VERSION:  $Id $
-- CREATED:  Tue Jan 23 10:51:45 CET 2001
-- AUTHOR:   Ralf Reissing
--
-- DESCRIPTION:
--   Abstract data object for tokenizing an Ada program. The
--   procedure Get_Token returns the tokens
-- IMPLEMENTATION:
--   The OpenToken package is used for lexical analysis
--   The Ada_Lexer module that comes together with OpenToken
--   is used to do the Ada lexical analysis
--   Input comes from an OpenToken.Text_Feeder.Text_IO instance,
--   therefore Ada.Text_IO input routines cannot be used between
--   calls to Open_Input and Close_Input!
--
-- MODIFICATIONS:
------------------------------------------------------------------------------

with OpenToken.Text_Feeder.Text_IO;
with Ada_Lexer; use Ada_Lexer;
with Ada.Text_IO;
with Ada.Containers.Vectors;

package body Ada_Tokenizer is

   type Token_Storage_Type is record
      Kind  : Token_Type;
      Value : Unbounded_String;
      Line  : Positive;
   end record;

   function Compare_Storage_Tokens
     (Left  : in Token_Storage_Type;
      Right : in Token_Storage_Type) return Boolean is
   begin
      return Left.Kind = Right.Kind and then
        "=" (Left.Value, Right.Value) and then
	Left.Line = Right.Line;
   end Compare_Storage_Tokens;
   
   package Token_Vectors is new Ada.Containers.Vectors
     (Index_Type   => Natural,
      Element_Type => Token_Storage_Type,
      "="          => Compare_Storage_Tokens);
   use Token_Vectors;
   
   Lookahead_Buffer : Token_Vectors.Vector := Token_Vectors.Empty_Vector;
   
  Analyzer: Tokenizer.Instance;

  -- Global text file for reading parse data
  File: Ada.Text_IO.File_Type;

  ----------------------------------------------------------------------------
  -- PROCEDURE: Open_Input
  --
  -- DESCRIPTION:
  --   Opens the file filename and returns whether the file could be opened
  --   successfully
  -- IMPLEMENTATION:
  --   Initializes the tokenizer and reads the first token
  --   This token is needed for Token_Available to check for EOF
  --   Ada.Text_IO is set to read its input from the file
  procedure Open_Input (Filename: in     String;
                        Success :    out Boolean) is
  begin
    Ada.Text_IO.Open
      (File => File,
       Mode => Ada.Text_IO.In_File,
       Name => Filename);
    Ada.Text_IO.Set_Input (File);
    Tokenizer.Input_Feeder := OpenToken.Text_Feeder.Text_IO.Create;

    Ada_Lexer.Init_Analyzer;
    Analyzer := Ada_Lexer.Get_Analyzer;
    Tokenizer.Find_Next (Analyzer);

    Success := True;
  exception
    when others =>
      Success := False;
  end Open_Input;

  ----------------------------------------------------------------------------
  -- PROCEDURE: Close_Input
  --
  -- DESCRIPTION:
  --   Closes the file opened with OpenInput.
  -- IMPLEMENTATION:
  --   Resets Ada.Text_IO to read from standard input
  procedure Close_Input is
  begin
    Ada.Text_IO.Close(File);
    Ada.Text_IO.Set_Input(Ada.Text_IO.Standard_Input);
    Lookahead_Buffer.Clear;
  end Close_Input;

  ----------------------------------------------------------------------------
  -- FUNCTION: Token_Available
  --
  -- DESCRIPTION:
  --   Checks if a token can be read or is in the look ahead buffer
  function Token_Available
    return Boolean is
  begin
   return (not Lookahead_Buffer.Is_Empty) or else
     (Tokenizer.ID (Analyzer) /= End_Of_File_T);
  end Token_Available;

  ----------------------------------------------------------------------------
  -- FUNCTION: Token_Readable
  --
  -- DESCRIPTION:
  --   Checks if a token can be read
  function Token_Readable
    return Boolean is
  begin
   return Tokenizer.ID (Analyzer) /= End_Of_File_T;
  end Token_Readable;

  ----------------------------------------------------------------------------
  -- PROCEDURE: Read_Token
  procedure Read_Token (Kind : out Token_Type;
                   Value: out Unbounded_String;
                   Line : out Positive) is
  begin
   Kind  := Token_Type(Tokenizer.ID (Analyzer));
   Value := Ada.Strings.Unbounded.To_Unbounded_String(
	      Tokenizer.Lexeme (Analyzer));
   Line  := OpenToken.Text_Feeder.Text_IO.Current_Line (
	      Tokenizer.Input_Feeder);
   Tokenizer.Find_Next (Analyzer);
  end Read_Token;
  
  ----------------------------------------------------------------------------
  -- PROCEDURE: Get_Token
  --
  -- DESCRIPTION:
  --   Returns information about the current token
  --   AND advances the file pointer to the next token.
  procedure Get_Token (Kind : out Token_Type;
                   Value: out Unbounded_String;
                   Line : out Positive) is
   Token : Token_Storage_Type;
  begin
   if not Lookahead_Buffer.Is_Empty then
      Token := Lookahead_Buffer.Element (Lookahead_Buffer.First_Index);
      Lookahead_Buffer.Delete_First;
      Kind  := Token.Kind;
      Value := Token.Value;
      Line  := Token.Line;
   else
      Read_Token (Kind, Value, Line);
   end if;
  end Get_Token;

  ----------------------------------------------------------------------------
  -- PROCEDURE: Look_Ahead
  procedure Look_Ahead
     (Kind   : out Token_Type;
      Value  : out Unbounded_String;
      Line   : out Positive;
      Success: out Boolean;
      Tokens : in Positive := 1) is
      
      Token : Token_Storage_Type;
      
      Lookahead : Natural;
      
      Buffered_Tokens : Integer := Integer (Lookahead_Buffer.Length);
  begin
   Success := False;
   if Buffered_Tokens >= Tokens then
      Token   := Lookahead_Buffer.Element (
        Lookahead_Buffer.First_Index + (Tokens - 1));
      Kind    := Token.Kind;
      Value   := Token.Value;
      Line    := Token.Line;
      Success := True;
   else
      Lookahead := Tokens - Buffered_Tokens;
      while Token_Readable and then Lookahead > 0 loop
	 Read_Token (Kind, Value, Line);
	 Token.Kind  := Kind;
	 Token.Value := Value;
	 Token.Line  := Line;
	 Lookahead_Buffer.Append (Token);
	 Lookahead := Lookahead - 1;
      end loop;
      if Lookahead = 0 then
	 Success := True;
      end if;
   end if;
  end Look_Ahead;
  
end Ada_Tokenizer;
